package com.jinghang.spark_base._020_SQL

import org.apache.spark.sql.SparkSession

object _020_DatasetCreationExample {

  //case class 不用new ，一般声明数据结构时使用
  case class Person(name: String, age: Long)

  def main(args: Array[String]): Unit = {

    val sparkSession = SparkSession.builder()
      .appName("_010_BasicDataFrameExample")
      .master("local[1]")
      .getOrCreate()

    sparkSession.sparkContext.setLogLevel("ERROR")

    runDatasetCreationExample(sparkSession)
  }
  private def runDatasetCreationExample(spark: SparkSession): Unit = {
    // $example on:create_ds$
    // Encoders are created for case classes
    import spark.implicits._
    //Seq是一个集合，默认被List实现,可以把他理解为List
    val caseClassDS = Seq(Person("Andy", 32)).toDS()
    val caseClassDF = Seq(Person("Andy", 32)).toDF()
    caseClassDS.show()
    // +----+---+
    // |name|age|
    // +----+---+
    // |Andy| 32|
    // +----+---+

    // Encoders for most common types are automatically provided by importing spark.implicits._
    val primitiveDS = Seq(1, 2, 3).toDS()
    primitiveDS.map(_ + 1).collect() // Returns: Array(2, 3, 4)

    // DataFrames can be converted to a Dataset by providing a class. Mapping will be done by name
    val path = "data/practiceOperator/people.json"
    val peopleDS = spark.read.json(path).as[Person]
    peopleDS.show()
    // +----+-------+
    // | age|   name|
    // +----+-------+
    // |null|Michael|
    // |  30|   Andy|
    // |  19| Justin|
    // +----+-------+
    // $example off:create_ds$
    peopleDS.select("name").show()
  }

}
